R Notebook

Replication of Goldman et al.

library(ggplot2)
library(tidyverse)

── Attaching core tidyverse packages ───────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ lubridate 1.9.4     ✔ tibble    3.2.1
✔ purrr     1.0.2     ✔ tidyr     1.3.1── Conflicts ─────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors

library(car)

Loading required package: carData

Attaching package: ‘car’

The following object is masked from ‘package:dplyr’:

    recode

The following object is masked from ‘package:purrr’:

    some

library(mgcv)

Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.9-1. For overview type 'help("mgcv-package")'.

require(gam)

Loading required package: gam
Loading required package: splines
Loading required package: foreach

Attaching package: ‘foreach’

The following objects are masked from ‘package:purrr’:

    accumulate, when

Loaded gam 1.22-5


Attaching package: ‘gam’

The following objects are masked from ‘package:mgcv’:

    gam, gam.control, gam.fit, s

require(lmtest)

Loading required package: lmtest
Loading required package: zoo

Attaching package: ‘zoo’

The following objects are masked from ‘package:base’:

    as.Date, as.Date.numeric

require(lme4)

Loading required package: lme4
Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack


Attaching package: ‘lme4’

The following object is masked from ‘package:nlme’:

    lmList

Load in & Preprocess the Data

df<-read.csv("replication_final.csv")
df$Family = str_to_title(str_replace(df$Family, "_", " "))

CHRTRM <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = CHR.TRM_SIGMORPHON_test_acc, 
           Goldman_acc = CHR.TRM_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw, 
           SIGMORPHON_train_size, 
           SIGMORPHON_train_lemmas
           ))

CHRTRM$Model = "CHR-TRM"



CLUZH <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = CLUZH_SIGMORPHON_test_acc, 
           Goldman_acc = CLUZH_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw,
           SIGMORPHON_train_size,
           SIGMORPHON_train_lemmas
           ))


CLUZH$Model = "CLUZH"



DeepSpin <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = DeepSpin_SIGMORPHON_test_acc, 
           Goldman_acc = DeepSpin_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw,
           SIGMORPHON_train_size,
           SIGMORPHON_train_lemmas
           ))

DeepSpin$Model = "DeepSpin"



all_data <- rbind(CLUZH, CHRTRM, DeepSpin)



all_data$acc_drop = all_data$Goldman_acc - all_data$SIGMORPHON_acc
all_data$log_acc_drop <- -1*log(-1*(all_data$acc_drop - 4))
max(all_data$acc_drop)

[1] 2.0202

# We'll use this helper function to run our stats
correlations <- function(a, b){
  for (m in list("pearson", "spearman", "kendall")){
    # Supressing warnings bc we'll get them whenever there are ties 
    suppressWarnings(res <- cor.test(a, b, method = m))
    formatted <- sprintf("%s: %f (p = %f)", res$method, res$estimate, res$p.value)
    print(formatted)
  }
}

# We'll use this helper function to get information about our model
eval_model <- function(model, df){
  rsquared = summary(model)$r.squared
  AIC = AIC(model)
  results <- sprintf("R^2: %f, AIC: %f", rsquared, AIC)
  print(results)
  layout(matrix(c(1,2,3,4),2,2)) 
  plot(model)
  return(predict(model, df, se = TRUE))
}

# We'll use this helper function to compare two models 
compare_models <- function(model_both, model_single){
  layout(matrix(c(1,2,3,4),2,2)) 
  plot(model_both)
  AIC = AIC(model_both)
  anovap = anova(model_both, model_single)$`Pr(>F)`[-1]
  results <- sprintf("AIC: %f, ANOVA p: %f", AIC, anovap)
  print(results)
  vif(model_both)
}

Raw Accuracy

Initial Scatters

SIGMORPHON <- all_data %>% 
  select(c(Model, Family, train_size=SIGMORPHON_train_size, train_lemmas=SIGMORPHON_train_lemmas, test_acc = SIGMORPHON_acc))
SIGMORPHON$Data <- "SIGMORPHON" 

Goldman <- all_data %>%
  select(c(Model, Family, train_size=Goldman_train_size, train_lemmas=Goldman_train_lemmas, test_acc = Goldman_acc))
Goldman$Data <- "Goldman et al."

rbind(SIGMORPHON, Goldman) %>%
  ggplot(aes(log(train_size), log(test_acc), color=Data)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
  scale_color_manual(values=c("turquoise", "purple")) +
  geom_point(aes(shape=Family), size=3, alpha=0.6) + 
  theme_bw() + 
  xlab("Training Size, Log Scale") + 
  ylab("Test Accuracy, Log Scale") + 
  ggtitle("Test Accuracy vs. Training Size") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_size_accuracy.pdf", device="pdf", width = 12, height=4)

rbind(SIGMORPHON, Goldman) %>%
  ggplot(aes(log(train_lemmas), log(test_acc), color=Data)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
  scale_color_manual(values=c("turquoise", "purple")) +
  geom_point(aes(shape=Family), size=3, alpha=0.6) + 
  theme_bw() + 
  xlab("Training Lemmas, Log Scale") + 
  ylab("Test Accuracy, Log Scale") + 
  ggtitle("Test Accuracy vs. Training Lemmas") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_lemmas_accuracy.pdf", device="pdf", width = 12, height=4)

Basic Correlational Stats

correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$Goldman_acc + 1))

[1] "Pearson's product-moment correlation: 0.703670 (p = 0.000003)"
[1] "Spearman's rank correlation rho: 0.817515 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.621986 (p = 0.000000)"

sprintf("")

[1] ""

correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$Goldman_acc + 1))

[1] "Pearson's product-moment correlation: 0.411709 (p = 0.015553)"
[1] "Spearman's rank correlation rho: 0.028615 (p = 0.872373)"
[1] "Kendall's rank correlation tau: -0.012590 (p = 0.917200)"

sprintf("")

[1] ""

correlations(log(CLUZH$Goldman_train_size), log(CLUZH$Goldman_acc + 1))

[1] "Pearson's product-moment correlation: 0.509641 (p = 0.002079)"
[1] "Spearman's rank correlation rho: 0.062892 (p = 0.723823)"
[1] "Kendall's rank correlation tau: 0.019785 (p = 0.870228)"

correlations(log(CHRTRM$SIGMORPHON_train_size), log(CHRTRM$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.308926 (p = 0.075448)"
[1] "Spearman's rank correlation rho: 0.039633 (p = 0.823893)"
[1] "Kendall's rank correlation tau: 0.008993 (p = 0.940805)"

sprintf("")

[1] ""

correlations(log(DeepSpin$SIGMORPHON_train_size), log(DeepSpin$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.318779 (p = 0.066132)"
[1] "Spearman's rank correlation rho: -0.028388 (p = 0.873380)"
[1] "Kendall's rank correlation tau: -0.058140 (p = 0.633569)"

sprintf("")

[1] ""

correlations(log(CLUZH$SIGMORPHON_train_size), log(CLUZH$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.401506 (p = 0.018593)"
[1] "Spearman's rank correlation rho: 0.120893 (p = 0.495836)"
[1] "Kendall's rank correlation tau: 0.078642 (p = 0.514054)"

correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$Goldman_acc+1))

[1] "Pearson's product-moment correlation: 0.663836 (p = 0.000019)"
[1] "Spearman's rank correlation rho: 0.849805 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.646953 (p = 0.000000)"

sprintf("")

[1] ""

correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$Goldman_acc+1))

[1] "Pearson's product-moment correlation: 0.239558 (p = 0.172389)"
[1] "Spearman's rank correlation rho: -0.011326 (p = 0.949309)"
[1] "Kendall's rank correlation tau: -0.057711 (p = 0.634496)"

sprintf("")

[1] ""

correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$Goldman_acc+1))

[1] "Pearson's product-moment correlation: 0.381134 (p = 0.026152)"
[1] "Spearman's rank correlation rho: 0.061529 (p = 0.729582)"
[1] "Kendall's rank correlation tau: -0.010821 (p = 0.928971)"

correlations(log(CHRTRM$SIGMORPHON_train_lemmas), log(CHRTRM$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.155920 (p = 0.378556)"
[1] "Spearman's rank correlation rho: -0.002142 (p = 0.990405)"
[1] "Kendall's rank correlation tau: -0.057608 (p = 0.634570)"

sprintf("")

[1] ""

correlations(log(DeepSpin$SIGMORPHON_train_lemmas), log(DeepSpin$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.161117 (p = 0.362669)"
[1] "Spearman's rank correlation rho: -0.084248 (p = 0.635701)"
[1] "Kendall's rank correlation tau: -0.118201 (p = 0.332834)"

sprintf("")

[1] ""

correlations(log(CLUZH$SIGMORPHON_train_lemmas), log(CLUZH$SIGMORPHON_acc+1))

[1] "Pearson's product-moment correlation: 0.268885 (p = 0.124126)"
[1] "Spearman's rank correlation rho: 0.095224 (p = 0.592169)"
[1] "Kendall's rank correlation tau: 0.030411 (p = 0.800929)"

GLMer Attempt

data <- rbind(SIGMORPHON, Goldman)
data$scaled_acc <- (data$test_acc/100 * 203 + .5) / 204


model <- glmer(scaled_acc ~ log(train_lemmas) + log(train_size) + (1|Model)  + ( train_size + train_lemmas |Family) + (train_lemmas |Data), data=data, family=binomial)
summary(model)

Beta Regression

rbind(CLUZH, CHRTRM, DeepSpin) %>%
  ggplot(aes(x = Goldman_scaled_acc, color=Model)) + 
  geom_density(aes(fill= Model), alpha = 0.5) + 
  scale_color_manual(values=c("purple", "gold", "turquoise")) +
  scale_fill_manual(values=c("purple", "gold", "turquoise")) +
  theme_bw() + 
  ylab("Density") + 
  xlab("Goldman et al. Test Accuracy") + 
  ggtitle("Density of Goldman et al. Test Accuracy") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/accuracy_distribution.pdf", device="pdf", width = 8, height=5)

CHRTRM$Goldman_scaled_acc <- (CHRTRM$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.4786 -0.8088  0.0000  0.2771  1.6071 

(Dispersion Parameter for Beta regression family taken to be 0.3493)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 1.0581 on 32 degrees of freedom
AIC: -3694.535 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1 19.689 19.6892   56.36 1.51e-08 ***
Residuals               32 11.179  0.3493                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.9217 -0.8786  0.0000  0.0000  1.1945 

(Dispersion Parameter for Beta regression family taken to be 0.3355)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 12.528 on 32 degrees of freedom
AIC: -3798.311 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 13.168 13.1679  39.247 5.054e-07 ***
Residuals                 32 10.736  0.3355                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.7181 -0.9309  0.0000  0.0000  1.3871 

(Dispersion Parameter for Beta regression family taken to be 0.3214)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 9.2018 on 31 degrees of freedom
AIC: -3878.832 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 14.2329 14.2329 44.2893 1.932e-07 ***
log(Goldman_train_size)    1  0.7122  0.7122  2.2163    0.1467    
Residuals                 31  9.9622  0.3214                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

AIC(size_model)

[1] -3694.535

AIC(lemma_model)

[1] -3798.311

AIC(both_model)

[1] -3878.832

BIC(size_model)

[1] -3691.483

BIC(lemma_model)

[1] -3795.258

BIC(both_model)

[1] -3874.253

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
             -4.9490885               0.6433317 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
               -3.8351949                 0.8111991 

DeepSpin$Goldman_scaled_acc <- (DeepSpin$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5926 -1.0797  0.0000  0.0000  0.8346 

(Dispersion Parameter for Beta regression family taken to be 0.3122)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.9009 on 32 degrees of freedom
AIC: -12273.43 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value  Pr(>F)   
log(Goldman_train_size)  1 3.5977  3.5977  11.522 0.00185 **
Residuals               32 9.9918  0.3122                   
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit

lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6147 -1.0699  0.0000  0.0000  0.5571 

(Dispersion Parameter for Beta regression family taken to be 0.3817)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -17.41 on 32 degrees of freedom
AIC: -12163.35 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value  Pr(>F)  
log(Goldman_train_lemmas)  1  1.4442 1.44423  3.7841 0.06057 .
Residuals                 32 12.2131 0.38166                  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit

both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) + 
    log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6214 -0.9821  0.0000  0.0000  0.7569 

(Dispersion Parameter for Beta regression family taken to be 0.3064)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.6952 on 31 degrees of freedom
AIC: -12350.04 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_size)    1 3.5682  3.5682 11.6462 0.001809 **
log(Goldman_train_lemmas)  1 0.8896  0.8896  2.9036 0.098388 . 
Residuals                 31 9.4980  0.3064                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

AIC(size_model)

[1] -12273.43

AIC(lemma_model)

[1] -12163.35

AIC(both_model)

[1] -12350.04

BIC(size_model)

[1] -12270.37

BIC(lemma_model)

[1] -12160.3

BIC(both_model)

[1] -12345.47

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
             -0.6355997               0.2861804 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
                0.6611814                 0.1920062 

CLUZH$Goldman_scaled_acc <- (CLUZH$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6764 -0.9117  0.0000  0.0000  0.7096 

(Dispersion Parameter for Beta regression family taken to be 0.4547)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -10.9053 on 32 degrees of freedom
AIC: -10671.66 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df  Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1  7.0202  7.0202   15.44 0.000427 ***
Residuals               32 14.5500  0.4547                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5490 -1.0794  0.0000  0.0000  0.8328 

(Dispersion Parameter for Beta regression family taken to be 0.5099)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -12.5557 on 32 degrees of freedom
AIC: -10473.82 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_lemmas)  1  4.9185  4.9185  9.6466 0.003957 **
Residuals                 32 16.3160  0.5099                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) + 
    log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6752 -0.8953  0.0000  0.0000  0.7093 

(Dispersion Parameter for Beta regression family taken to be 0.4671)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -11.0956 on 31 degrees of freedom
AIC: -10675.54 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_size)    1  7.0203  7.0203 15.0303 0.0005137 ***
log(Goldman_train_lemmas)  1  0.0050  0.0050  0.0107 0.9182216    
Residuals                 31 14.4794  0.4671                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

AIC(size_model)

[1] -10671.66

AIC(lemma_model)

[1] -10473.82

AIC(both_model)

[1] -10675.54

BIC(size_model)

[1] -10668.61

BIC(lemma_model)

[1] -10470.76

BIC(both_model)

[1] -10670.96

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
             -1.7996408               0.3963434 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
               -0.5208229                 0.3695156 

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_size, Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_size, size_fit)) + 
  geom_ribbon(aes(x = Goldman_train_size, 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_size_beta.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_lemmas, Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_lemmas, lemma_fit)) + 
  geom_ribbon(aes(x = Goldman_train_lemmas, 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

#ggsave("figures/train_lemmas_beta.pdf", device="pdf", width = 12, height=4)

Accuracy Drop

Initial Visualization

Raw Plots

all_data %>% 
  ggplot(aes(x = Goldman_train_size, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Size") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Size vs. Accuracy Drop") 

all_data %>% 
  ggplot(aes(x = Goldman_train_lemmas, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Lemmas") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Lemmas vs. Accuracy Drop")

all_data %>% 
  ggplot(aes(x = train_lemma_diff_raw, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Drop in the Number of Training Lemmas") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Lemma Drop vs. Accuracy Drop")

Log-Log Plots and LM Fit Lines

all_data %>% 
  ggplot(aes(x = log(Goldman_train_size), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Size, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Training Size vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

all_data %>% 
  ggplot(aes(x = log(Goldman_train_lemmas), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Lemmas, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Training Lemmas vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

all_data %>% 
  ggplot(aes(x = log(-1*train_lemma_diff_raw), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Drop in Training Lemmas, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Lemma Drop vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

Basic Correlational Stats

CHRTRM$scaled_drop <- -1*(CHRTRM$Goldman_acc - CHRTRM$SIGMORPHON_acc - 2)/100
DeepSpin$scaled_drop <- -1*(DeepSpin$Goldman_acc - DeepSpin$SIGMORPHON_acc - 2)/100
CLUZH$scaled_drop <- -1*(CLUZH$Goldman_acc - CLUZH$SIGMORPHON_acc - 4)/100

correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$scaled_drop))

[1] "Pearson's product-moment correlation: -0.830738 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.798319 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.593583 (p = 0.000000)"

sprintf("")

[1] ""

correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$scaled_drop))

[1] "Pearson's product-moment correlation: -0.125834 (p = 0.478255)"
[1] "Spearman's rank correlation rho: -0.032631 (p = 0.854639)"
[1] "Kendall's rank correlation tau: -0.010841 (p = 0.928903)"

sprintf("")

[1] ""

correlations(log(CLUZH$Goldman_train_size), log(CLUZH$scaled_drop))

[1] "Pearson's product-moment correlation: -0.233565 (p = 0.183703)"
[1] "Spearman's rank correlation rho: -0.130491 (p = 0.461985)"
[1] "Kendall's rank correlation tau: -0.067797 (p = 0.573169)"

correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$scaled_drop))

[1] "Pearson's product-moment correlation: -0.907756 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.915412 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.747098 (p = 0.000000)"

sprintf("")

[1] ""

correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$scaled_drop))

[1] "Pearson's product-moment correlation: -0.095421 (p = 0.591397)"
[1] "Spearman's rank correlation rho: -0.080678 (p = 0.650137)"
[1] "Kendall's rank correlation tau: -0.052539 (p = 0.666183)"

sprintf("")

[1] ""

correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$scaled_drop))

[1] "Pearson's product-moment correlation: -0.264233 (p = 0.131023)"
[1] "Spearman's rank correlation rho: -0.196087 (p = 0.266378)"
[1] "Kendall's rank correlation tau: -0.137746 (p = 0.253461)"

correlations(log(-1*CHRTRM$train_lemma_diff_raw), log(CHRTRM$scaled_drop))

[1] "Pearson's product-moment correlation: -0.896908 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.888906 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.726787 (p = 0.000000)"

sprintf("")

[1] ""

correlations(log(-1*DeepSpin$train_lemma_diff_raw), log(DeepSpin$scaled_drop))

[1] "Pearson's product-moment correlation: -0.146380 (p = 0.408762)"
[1] "Spearman's rank correlation rho: -0.128783 (p = 0.467918)"
[1] "Kendall's rank correlation tau: -0.086884 (p = 0.475256)"

sprintf("")

[1] ""

correlations(log(-1*CLUZH$train_lemma_diff_raw), log(CLUZH$scaled_drop))

[1] "Pearson's product-moment correlation: -0.281993 (p = 0.106145)"
[1] "Spearman's rank correlation rho: -0.236036 (p = 0.178977)"
[1] "Kendall's rank correlation tau: -0.157283 (p = 0.191899)"

Beta Regression

rbind(CLUZH, CHRTRM, DeepSpin) %>%
  ggplot(aes(x = scaled_drop, color=Model)) + 
  geom_density(aes(fill= Model), alpha = 0.5) + 
  scale_color_manual(values=c("purple", "gold", "turquoise")) +
  scale_fill_manual(values=c("purple", "gold", "turquoise")) +
  theme_bw() + 
  ylab("Density") + 
  xlab("Accuracy Drop") + 
  ggtitle("Density of Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/drop_distribution.pdf", device="pdf", width = 8, height=5)

size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.3926 -0.3985  0.0000  0.8903  1.0690 

(Dispersion Parameter for Beta regression family taken to be 0.3693)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: -3.3233 on 32 degrees of freedom
AIC: 1550.002 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_size)  1 15.365 15.3647   41.61 2.964e-07 ***
Residuals               32 11.816  0.3693                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.1012  0.0000  0.6957  1.0132  1.9912 

(Dispersion Parameter for Beta regression family taken to be 0.1745)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 16.7735 on 32 degrees of freedom
AIC: 1016.286 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 11.8181 11.8181  67.719 2.119e-09 ***
Residuals                 32  5.5846  0.1745                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CHRTRM)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5036  0.0000  0.4835  0.9504  1.5863 

(Dispersion Parameter for Beta regression family taken to be 0.2471)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 8.804 on 32 degrees of freedom
AIC: 1193.401 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                               Df  Sum Sq Mean Sq F value    Pr(>F)    
log(-1 * train_lemma_diff_raw)  1 12.6416 12.6416  51.154 4.056e-08 ***
Residuals                      32  7.9081  0.2471                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(drop_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$drop_fit <- res$fit
CHRTRM$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas)  + log(-1* train_lemma_diff_raw) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 * 
    train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.0309  0.0000  0.6883  1.1058  2.1611 

(Dispersion Parameter for Beta regression family taken to be 0.1944)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 20.2587 on 30 degrees of freedom
AIC: 956.8627 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                               Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)       1 10.9621 10.9621 56.3870 2.267e-08 ***
log(-1 * train_lemma_diff_raw)  1  0.2024  0.2024  1.0412    0.3157    
log(Goldman_train_size)         1  0.1658  0.1658  0.8530    0.3631    
Residuals                      30  5.8323  0.1944                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
              3.9535916              -0.5549763 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
                3.9757535                -0.9047767 

drop_model$coefficients

                   (Intercept) log(-1 * train_lemma_diff_raw) 
                     2.8158287                     -0.8161564 

AIC(size_model)

[1] 1550.002

AIC(lemma_model)

[1] 1016.286

AIC(drop_model)

[1] 1193.401

BIC(size_model)

[1] 1553.055

BIC(lemma_model)

[1] 1019.338

BIC(drop_model)

[1] 1196.454

size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.7397  0.0000  0.0000  1.1084  1.4802 

(Dispersion Parameter for Beta regression family taken to be 0.1433)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.4806 on 32 degrees of freedom
AIC: 7018.613 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_size)  1 1.8557 1.85567  12.945 0.001067 **
Residuals               32 4.5871 0.14335                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.5064  0.0000  0.0000  1.0176  1.5635 

(Dispersion Parameter for Beta regression family taken to be 0.1553)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -14.7622 on 32 degrees of freedom
AIC: 7027.519 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_lemmas)  1 1.1735 1.17353  7.5547 0.009755 **
Residuals                 32 4.9708 0.15534                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = DeepSpin)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.5698  0.0000  0.0000  1.0924  1.5544 

(Dispersion Parameter for Beta regression family taken to be 0.1475)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.6903 on 32 degrees of freedom
AIC: 6992.3 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value   Pr(>F)   
log(-1 * train_lemma_diff_raw)  1 1.5505  1.5505  10.512 0.002771 **
Residuals                      32 4.7199  0.1475                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(drop_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$drop_fit <- res$fit
DeepSpin$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(-1*train_lemma_diff_raw) + log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw) + 
    log(Goldman_train_size) + log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.8752  0.0000  0.0000  0.9393  1.5220 

(Dispersion Parameter for Beta regression family taken to be 0.1435)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.7918 on 30 degrees of freedom
AIC: 6982.896 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value   Pr(>F)   
log(-1 * train_lemma_diff_raw)  1 1.6345 1.63447 11.3933 0.002052 **
log(Goldman_train_size)         1 0.1767 0.17666  1.2314 0.275945   
log(Goldman_train_lemmas)       1 0.3227 0.32272  2.2495 0.144106   
Residuals                      30 4.3038 0.14346                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
             -0.2192908              -0.2378525 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
               -1.0371551                -0.2098211 

drop_model$coefficients

                   (Intercept) log(-1 * train_lemma_diff_raw) 
                    -1.0666528                     -0.2533969 

AIC(size_model)

[1] 7018.613

AIC(lemma_model)

[1] 7027.519

AIC(drop_model)

[1] 6992.3

BIC(size_model)

[1] 7021.666

BIC(lemma_model)

[1] 7030.572

BIC(drop_model)

[1] 6995.352

size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
   Min     1Q Median     3Q    Max 
-0.838  0.000  0.000  1.005  1.417 

(Dispersion Parameter for Beta regression family taken to be 0.1677)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 1.805 on 32 degrees of freedom
AIC: 5855.958 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1 2.2528 2.25278  13.431 0.000889 ***
Residuals               32 5.3675 0.16774                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.6502  0.0000  0.3396  1.0907  1.4213 

(Dispersion Parameter for Beta regression family taken to be 0.1501)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.7868 on 32 degrees of freedom
AIC: 5823.695 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 2.5131 2.51310  16.738 0.0002709 ***
Residuals                 32 4.8047 0.15015                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CLUZH)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
   Min     1Q Median     3Q    Max 
-0.841  0.000  0.000  1.094  1.376 

(Dispersion Parameter for Beta regression family taken to be 0.1516)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.8613 on 32 degrees of freedom
AIC: 5831.558 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value    Pr(>F)    
log(-1 * train_lemma_diff_raw)  1 2.5820 2.58196  17.027 0.0002453 ***
Residuals                      32 4.8526 0.15164                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

res <- predict(drop_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$drop_fit <- res$fit
CLUZH$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas) + log(-1*train_lemma_diff_raw) + log(Goldman_train_size) , family=betar(link="logit"), data = CLUZH)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 * 
    train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.7097  0.0000  0.3350  1.0758  1.4194 

(Dispersion Parameter for Beta regression family taken to be 0.1584)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 4.0005 on 30 degrees of freedom
AIC: 5837.224 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)       1 2.5226 2.52261 15.9285 0.0003913 ***
log(-1 * train_lemma_diff_raw)  1 0.0524 0.05242  0.3310 0.5693799    
log(Goldman_train_size)         1 0.0128 0.01279  0.0808 0.7782086    
Residuals                      30 4.7511 0.15837                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

size_model$coefficients

            (Intercept) log(Goldman_train_size) 
              0.1286658              -0.2387694 

lemma_model$coefficients

              (Intercept) log(Goldman_train_lemmas) 
                -0.328039                 -0.287939 

drop_model$coefficients

                   (Intercept) log(-1 * train_lemma_diff_raw) 
                    -0.5445538                     -0.3002098 

AIC(size_model)

[1] 5855.958

AIC(lemma_model)

[1] 5823.695

AIC(drop_model)

[1] 5831.558

BIC(size_model)

[1] 5859.011

BIC(lemma_model)

[1] 5826.747

BIC(drop_model)

[1] 5834.611

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_size, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_size, -1*size_fit)) + 
  geom_ribbon(aes(x = Goldman_train_size, 
                  ymin = -1*(size_fit - 2 * size_se),
                  ymax = -1*(size_fit + 2 * size_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size") + 
  ylab("Scaled Accuracy Drop") + 
  ggtitle("Training Size vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_size_drop.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_lemmas, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_lemmas, -1*lemma_fit)) + 
  geom_ribbon(aes(x = Goldman_train_lemmas, 
                  ymin = -1*(lemma_fit - 2 * lemma_se),
                  ymax = -1*(lemma_fit + 2 * lemma_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas") + 
  ylab("Scaled Accuracy Drop") +
  ggtitle("Training Lemmas vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_lemmas_drop.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(-1*train_lemma_diff_raw, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(-1*train_lemma_diff_raw, -1*drop_fit)) + 
  geom_ribbon(aes(x = -1*train_lemma_diff_raw, 
                  ymin = -1*(drop_fit - 2 * drop_se),
                  ymax = -1*(drop_fit + 2 * drop_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Drop in Lemmas from SIGMORPHON to Goldman et al.") + 
  ylab("Scaled Accuracy Drop") +
  ggtitle("Training Lemma Drop vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/lemma_drop_drop.pdf", device="pdf", width = 12, height=4)

Other Regressions

Linear Regression: Raw Accuracy

size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = CHRTRM)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = CHRTRM)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.12468 -0.31146  0.01354  0.48987  1.86432 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)    
(Intercept)             -4.58098    0.65386  -7.006 6.12e-08 ***
log(Goldman_train_size)  0.42338    0.07346   5.763 2.16e-06 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9212 on 32 degrees of freedom
Multiple R-squared:  0.5093,    Adjusted R-squared:  0.494 
F-statistic: 33.22 on 1 and 32 DF,  p-value: 2.156e-06

res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = CHRTRM)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.5247 -0.5142  0.3007  0.7480  1.1648 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)               -3.21974    0.47613  -6.762 1.22e-07 ***
log(Goldman_train_lemmas)  0.40297    0.07828   5.148 1.29e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9727 on 32 degrees of freedom
Multiple R-squared:  0.453, Adjusted R-squared:  0.4359 
F-statistic:  26.5 on 1 and 32 DF,  p-value: 1.293e-05

both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = CHRTRM)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.18137 -0.36728  0.06201  0.54080  1.74301 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)                -4.3744     0.7339  -5.960 1.37e-06 ***
log(Goldman_train_lemmas)   0.1066     0.1656   0.644   0.5246    
log(Goldman_train_size)     0.3292     0.1641   2.006   0.0537 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9298 on 31 degrees of freedom
Multiple R-squared:  0.5158,    Adjusted R-squared:  0.4846 
F-statistic: 16.51 on 2 and 31 DF,  p-value: 1.312e-05

AIC(size_model)

[1] 94.84671

AIC(lemma_model)

[1] 98.5432

AIC(both_model)

[1] 96.39553

anova(both_model, lemma_model)

Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq     F  Pr(>F)  
1     31 26.799                             
2     32 30.276 -1   -3.4769 4.022 0.05371 .
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = DeepSpin)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.09702 -0.07001  0.01215  0.18962  0.42460 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)   
(Intercept)             -0.77218    0.22958  -3.363  0.00201 **
log(Goldman_train_size)  0.06590    0.02579   2.555  0.01558 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3234 on 32 degrees of freedom
Multiple R-squared:  0.1694,    Adjusted R-squared:  0.1435 
F-statistic: 6.527 on 1 and 32 DF,  p-value: 0.01558

res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.28791 -0.03124  0.06302  0.20218  0.30658 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)  
(Intercept)               -0.42363    0.16866  -2.512   0.0173 *
log(Goldman_train_lemmas)  0.03873    0.02773   1.396   0.1722  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3446 on 32 degrees of freedom
Multiple R-squared:  0.05744,   Adjusted R-squared:  0.02799 
F-statistic:  1.95 on 1 and 32 DF,  p-value: 0.1722

both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-0.89596 -0.05727  0.06613  0.13517  0.53952 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)               -0.96792    0.24663  -3.925  0.00045 ***
log(Goldman_train_lemmas) -0.10099    0.05567  -1.814  0.07936 .  
log(Goldman_train_size)    0.15516    0.05516   2.813  0.00844 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3125 on 31 degrees of freedom
Multiple R-squared:  0.2491,    Adjusted R-squared:  0.2007 
F-statistic: 5.143 on 2 and 31 DF,  p-value: 0.01178

AIC(size_model)

[1] 23.67421

AIC(lemma_model)

[1] 27.97414

AIC(both_model)

[1] 22.24389

anova(both_model, lemma_model)

Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq      F  Pr(>F)   
1     31 3.0265                               
2     32 3.7992 -1  -0.77262 7.9137 0.00844 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = CLUZH)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = CLUZH)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.01946 -0.16571 -0.01182  0.38359  0.86925 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)    
(Intercept)             -1.65335    0.40383  -4.094 0.000269 ***
log(Goldman_train_size)  0.15234    0.04537   3.358 0.002041 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.5689 on 32 degrees of freedom
Multiple R-squared:  0.2605,    Adjusted R-squared:  0.2374 
F-statistic: 11.27 on 1 and 32 DF,  p-value: 0.002041

res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = CLUZH)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.4300 -0.1431  0.1326  0.3918  0.6468 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)   
(Intercept)               -0.99545    0.29911  -3.328  0.00221 **
log(Goldman_train_lemmas)  0.11548    0.04918   2.348  0.02521 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.6111 on 32 degrees of freedom
Multiple R-squared:  0.147, Adjusted R-squared:  0.1203 
F-statistic: 5.514 on 1 and 32 DF,  p-value: 0.02521

both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = CLUZH)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.80791 -0.16141  0.01489  0.28101  0.99017 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)                -1.8593     0.4484  -4.147 0.000242 ***
log(Goldman_train_lemmas)  -0.1063     0.1012  -1.050 0.301858    
log(Goldman_train_size)     0.2463     0.1003   2.456 0.019855 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.568 on 31 degrees of freedom
Multiple R-squared:  0.2859,    Adjusted R-squared:  0.2399 
F-statistic: 6.206 on 2 and 31 DF,  p-value: 0.005408

AIC(size_model)

[1] 62.07644

AIC(lemma_model)

[1] 66.93316

AIC(both_model)

[1] 62.88837

anova(both_model, lemma_model)

Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq      F  Pr(>F)  
1     31 10.003                              
2     32 11.949 -1   -1.9462 6.0317 0.01986 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_size), log(Goldman_scaled_acc))) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_size), size_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_size), 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_size_lm.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_lemmas), log(Goldman_scaled_acc))) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_lemmas), 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_lemmas_lm.pdf", device="pdf", width = 12, height=4)

Logistic Regression: Raw Accuracy

size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = CHRTRM, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = CHRTRM)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)   
(Intercept)              -5.0645     1.9139  -2.646  0.00814 **
log(Goldman_train_size)   0.6567     0.2284   2.875  0.00404 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  6.3867  on 32  degrees of freedom
AIC: 31.886

Number of Fisher Scoring iterations: 5

res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CHRTRM, family=binomial)

Warning: non-integer #successes in a binomial glm!

res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = CHRTRM)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)  
(Intercept)                -3.8352     1.6005  -2.396   0.0166 *
log(Goldman_train_lemmas)   0.8112     0.3161   2.566   0.0103 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  5.6775  on 32  degrees of freedom
AIC: 28.439

Number of Fisher Scoring iterations: 5

both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = CHRTRM)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)  
(Intercept)                -4.5197     1.9676  -2.297   0.0216 *
log(Goldman_train_lemmas)   0.5324     0.5415   0.983   0.3255  
log(Goldman_train_size)     0.2584     0.4329   0.597   0.5507  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  5.3129  on 31  degrees of freedom
AIC: 30.767

Number of Fisher Scoring iterations: 5

AIC(size_model)

[1] 31.88575

AIC(lemma_model)

[1] 28.4393

AIC(both_model)

[1] 30.76683

anova(both_model, lemma_model)

Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     5.3129                     
2        32     5.6775 -1 -0.36453    0.546

size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = DeepSpin, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = DeepSpin)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)
(Intercept)              -0.5417     1.9087  -0.284    0.777
log(Goldman_train_size)   0.2811     0.2380   1.181    0.238

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 6.5801  on 32  degrees of freedom
AIC: 19.053

Number of Fisher Scoring iterations: 5

res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = DeepSpin, family=binomial)

Warning: non-integer #successes in a binomial glm!

res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = DeepSpin)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                 0.7911     1.3545   0.584    0.559
log(Goldman_train_lemmas)   0.1786     0.2453   0.728    0.466

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 7.5112  on 32  degrees of freedom
AIC: 21.031

Number of Fisher Scoring iterations: 5

both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = DeepSpin)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                -0.9913     1.9715  -0.503    0.615
log(Goldman_train_lemmas)  -0.3398     0.5003  -0.679    0.497
log(Goldman_train_size)     0.5596     0.4763   1.175    0.240

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 6.1213  on 31  degrees of freedom
AIC: 19.851

Number of Fisher Scoring iterations: 5

AIC(size_model)

[1] 19.05313

AIC(lemma_model)

[1] 21.03097

AIC(both_model)

[1] 19.85117

anova(both_model, lemma_model)

Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     6.1213                     
2        32     7.5112 -1    -1.39   0.2384

size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = CLUZH, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = CLUZH)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)  
(Intercept)              -1.8475     1.8112  -1.020   0.3077  
log(Goldman_train_size)   0.4071     0.2317   1.757   0.0789 .
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.9162  on 33  degrees of freedom
Residual deviance:  9.3231  on 32  degrees of freedom
AIC: 24.943

Number of Fisher Scoring iterations: 5

res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CLUZH, family=binomial)

Warning: non-integer #successes in a binomial glm!

res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = CLUZH)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                -0.5165     1.3129  -0.393    0.694
log(Goldman_train_lemmas)   0.3758     0.2575   1.460    0.144

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.916  on 33  degrees of freedom
Residual deviance: 10.291  on 32  degrees of freedom
AIC: 26.417

Number of Fisher Scoring iterations: 5

both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH, family=binomial)

Warning: non-integer #successes in a binomial glm!

summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = CLUZH)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)               -1.89248    1.89135  -1.001    0.317
log(Goldman_train_lemmas) -0.03862    0.48110  -0.080    0.936
log(Goldman_train_size)    0.43737    0.44328   0.987    0.324

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.9162  on 33  degrees of freedom
Residual deviance:  9.3167  on 31  degrees of freedom
AIC: 27.012

Number of Fisher Scoring iterations: 5

AIC(size_model)

[1] 24.94273

AIC(lemma_model)

[1] 26.41689

AIC(both_model)

[1] 27.01204

anova(both_model, lemma_model)

Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     9.3167                     
2        32    10.2914 -1 -0.97468   0.3235

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_size), Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_size), size_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_size), 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_size_logistic.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_lemmas), Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_lemmas), 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/train_lemmas_logistic.pdf", device="pdf", width = 12, height=4)